import copy
from datetime import datetime
import threading
import time
import math

from dateutil.parser import parse
from asyncio import run

from dataInsertDB import cxOracleDB
from log.clog import *

import requests
import re
from settingObj.settings import settings
from utils.sheettools import dataAdd, dataAppend
from utils.tools import EXDICT, CONCLUEDEDICT, MUSTEXDICT, DISTURBDICT, NULLFLAG, ENSUREFLAG, \
    DEVELOP_WINNING_BID_DATA, DEVELOP_BIDDING_DATA, DICT_WIN_BID_DATA, DICT_BIDDING_DATA, NOT_DICT_BIDDING_DATA, \
    NOT_DICT_WIN_BID_DATA, CONSTDATA
import asyncio
from PageAnalyse.singlePageAnalyse import singlePageAnalyse


# CCGP爬虫类
class ccgpcrawler(cxOracleDB,singlePageAnalyse):
    # 初始化定义
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.TYPEOFBIDDING = ['ZcyAnnouncement2', 'ZcyAnnouncement5', 'ZcyAnnouncement3001']
        # self.TYPEOFBIDDING = ['ZcyAnnouncement2']
        self.AREATYOE = ['520999', '529900', '520100', '520800', '520899', '010200', '520902', '520199',
                         '520198', '520193', '520183', '520181', '520123', '520122', '520121', '520115',
                         '520113', '520112', '520111', '520103', '520102']
        self.lock = threading.Lock()

    # 请求头定义
    def ccgpTotalPage(self, page=1, isProvince=True, districtCode=529900, categoryCode='ZcyAnnouncement2'):
        cookies = {
            '_zcy_log_client_uuid': '82c68120-778f-11ee-9b91-23d640c8beda',
        }

        headers = {
            'Accept': 'application/json, text/plain, */*',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Content-Type': 'application/json;charset=UTF-8',
            # 'Cookie': '_zcy_log_client_uuid=82c68120-778f-11ee-9b91-23d640c8beda',
            'Origin': 'http://www.ccgp-guizhou.gov.cn',
            'Proxy-Connection': 'keep-alive',
            'Referer': f'http://www.ccgp-guizhou.gov.cn/luban/category?isProvince={isProvince}&districtCode={districtCode}&parentId=190013&childrenCode=ZcyAnnouncement10016&utm=luban.luban-PC-42534.1069-pc-wsg-ArticlePurchaseNoticeList-front.1.39e3cc7077cf11ee89af09ec223aee86',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
            'X-Requested-With': 'XMLHttpRequest',
        }

        json_data = {
            'pageNo': page,
            'pageSize': 50,
            'categoryCode': categoryCode,
            'publishDateBegin': self.config.start_time,
            'publishDateEnd': self.config.end_time,
            # 这里的 districtCode 指的是 南明区白云区这样的区域代码
            # 源代码中的 districtCode 指向是贵阳市贵州省代码
            'districtCode': [
                districtCode,
            ],
            'leaf': 0,
            '_t': 1707102942000,
        }
        try:
            response = requests.post(
                'http://www.ccgp-guizhou.gov.cn/portal/category',
                cookies=cookies,
                headers=headers,
                json=json_data,
                verify=False,
            )
        finally:
            response = response.json()['result']
            return response

    # 异步爬取
    async def sinlgePageCrawler(self, articleId):
        urls = 'http://www.ccgp-guizhou.gov.cn/site/detail?parentId=190013&articleId=' + articleId + '&utm=site.site-PC-42603.1024-pc-wsg-secondLevelPage-front.1.b87929b0c3d611eeb241751c79198825'

        cookies = {
            '_zcy_log_client_uuid': '82c68120-778f-11ee-9b91-23d640c8beda',
        }

        headers = {
            'Accept': 'application/json, text/plain, */*',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            # 'Cookie': '_zcy_log_client_uuid=82c68120-778f-11ee-9b91-23d640c8beda',
            'Proxy-Connection': 'keep-alive',
            'Referer': f'http://www.ccgp-guizhou.gov.cn/site/detail?parentId=190013&articleId={articleId}&utm=site.site-PC-42603.1024-pc-wsg-secondLevelPage-front.1.b87929b0c3d611eeb241751c79198825',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
            'X-Requested-With': 'XMLHttpRequest',
        }

        params = {
            'articleId': f'{articleId}',
            'parentId': '190013',
            'timestamp': '1707118185',
        }

        response = requests.get(
            'http://www.ccgp-guizhou.gov.cn/portal/detail',
            params=params,
            cookies=cookies,
            headers=headers,
            verify=False,
        )
        try:
            res = response.json()['result']['data']
        except Exception as e:
            logger.exception('解析错误', e, urls)
        finally:
            return res, urls

    # 正则表达式统一入口
    def matchingJudgment(self, patt, text):
        '''
        :param patt: 正则表达式
        :param text: 文本
        :return: 匹配结果
        '''
        try:
            for i in patt.values():
                # 要注意 flags 模式
                if res := re.search(i, text, flags=re.MULTILINE | re.DOTALL):
                    resbody = res.group(1)
                    return resbody
            return ' - '
        except:
            resbody = ' '
            return resbody

    '''
        :description 判断目标内容是否存在于字典中 
    '''

    def isInDict(self, patt, text):
        try:
            # 使用具体的异常类型代替裸的except
            match = re.search(patt, text, flags=re.MULTILINE | re.DOTALL)
            if match:
                return True
            else:
                return False
        except re.error as e:
            # print(f"正则表达式错误: {e}")
            raise e

    # 招标金额匹配
    def matchBidAmount(self, text):
        patt = {'patt1': r'采购预算：.*?\>(\d+).*?元',
                'patt2': r'预算金额\(元\).*?\">(\d+)<\/samp>'}
        res = self.matchingJudgment(patt=patt, text=text)
        return res

    # 招标人匹配
    def senderMatch(self, text):
        patt = {'patt1': r'采购人：.*?>(.*?)<\/samp>',
                'patt2': r'采购单位名称：.*?>(.*?)<\/samp>',
                'patt3': r'采购人信息.*?>名 称.*?>(.*?)<'}
        res = self.matchingJudgment(patt=patt, text=text)
        return res

    # 中标人匹配
    def theWinningBidderIsMatched(self, text):
        patt = {'patt1': r'code-winningSupplierName.*?>(.*?)<\/td>'}
        res = self.matchingJudgment(patt=patt, text=text)
        return res

    # 中标金额匹配
    def tenderPrice(self, text):
        patt = {'patt1': r'总价形式报价:(\d+)',
                'patt2': r'投标报价:(\d+)',
                'patt3': r'中标（成交）金额:(\d+)',
                'patt3': r'最终报价:(\d+)',
                'patt4': r'code-summaryPrice.*?>.*?(\d+)'}
        res = self.matchingJudgment(patt=patt, text=text)
        return res

    # 单页面分析
    def singlePageAnalyse(self, winType, text):
        sender = self.senderMatch(text)
        if winType:
            winBidder = self.theWinningBidderIsMatched(text)
            money = self.tenderPrice(text)
            if money != '-':
                money = super().convert_to_ten_thousand_unit(money)
            return money, sender, winBidder
        else:
            money = self.matchBidAmount(text)
        if money != '-':
            money = super().convert_to_ten_thousand_unit(money)
        return money, sender

    # 时间转换
    def timeConvert(self, times):
        times = times / 1000
        times = time.strftime('%Y-%m-%d', time.localtime(int(times)))
        return times

    # CCGP 入口
    def ccgpMainAnalyse(self, biddingtype, area):
        DESIGN_DICT_FLAG = False
        demandDisclosure = False
        WinTheBidOrNot = False
        '''
            ZcyAnnouncement2 "采购需求公示" 190016
            ZcyAnnouncement5 "采购结果公告" 190026

            520999 "贵安新区本级"
            529900 "贵州省本级"
            520100 "贵阳市本级"
        :return:
        '''

        if biddingtype == 'ZcyAnnouncement5':
            WinTheBidOrNot = True
        else:
            WinTheBidOrNot = False
        if biddingtype == 'ZcyAnnouncement2':
            demandDisclosure = True
        time.sleep(self.config.sleep_time)
        # 读取总页数
        jsonData = self.ccgpTotalPage()
        # print(jsonData['data']['total'])
        # print(jsonData)
        try:
            if (totalpage := math.ceil(jsonData['data']['total'] / 50)) == 0:
                totalpage = + 1
        except ValueError as e:
            totalpage = 1
            # print(e)
        except ZeroDivisionError as e:
            totalpage = 1
            # print(e)
        for page in range(1, int(totalpage) + 1):
            # print(
            #     f'当前爬取区域为:{area} 当前爬取类型为:{biddingtype},当前爬取页数为:{page}'
            # )
            time.sleep(self.config.sleep_time)
            jsonData = self.ccgpTotalPage(page=page,
                                          districtCode=area,
                                          categoryCode=biddingtype)
            for item in jsonData['data']['data']:
                DICTTYPE = False
                datadict = dict()
                #
                try:
                    datadict['title'] = item['title']
                except:
                    datadict['title'] = NULLFLAG
                    # print(e)
                    raise e
                #
                try:
                    datadict['districtName'] = item['districtName']
                except:
                    datadict['districtName'] = NULLFLAG
                    # print(e)
                    raise e

                datadict['publishDate'] = item['publishDate']
                datadict['articleId'] = item['articleId']
                res, url = asyncio.run(self.sinlgePageCrawler(articleId=datadict['articleId']))
                breakType = False

                # 判断标题是否重复
                for saveTitleInClass in self.config.titleContainer:
                    if saveTitleInClass == item['title']:
                        breakType = True
                if breakType:
                    continue
                self.config.titleContainer.append(item['title'])

                # 跳出单轮条件
                outOfLoopCondition = False

                # 判断是否属于EXDICT项目,如果是进行跳过
                for key in EXDICT:
                    if key in datadict['title']:
                        outOfLoopCondition = True
                        break
                if outOfLoopCondition:
                    # print(f'当前标题为{datadict["title"]}属于EXDICT项目,进行跳过')
                    continue

                # 判断是否属于CONCLUEDEDICT项目,如果是进行保存
                for key in CONCLUEDEDICT:
                    if key in res['content']:
                        outOfLoopCondition = False

                # 判断是否属于MUSTEXDICT项目,如果是进行保存
                for key in MUSTEXDICT:
                    if key in res['content']:
                        outOfLoopCondition = True
                        break
                if outOfLoopCondition:
                    # print(f'当前标题为{datadict["title"]}属于MUSTEXDICT项目,进行跳过')
                    continue
                # 是否属于dict 项目,如果是进行保存
                saveTitle = item['title']
                # 进行标题的替换,处理模糊关键字
                for disturbkeyword in DISTURBDICT:
                    if disturbkeyword in item['title']:
                        item['title'] = item['title'].replace(disturbkeyword, '')
                # 进行标题的替换,处理模糊关键字
                for save in CONCLUEDEDICT:
                    datadict['DICT'] = NULLFLAG
                    datadict['keyword'] = NULLFLAG
                    # if save in res['content']:
                    if save in item['title']:
                        outOfLoopCondition = False
                        DICTTYPE = True
                        datadict['DICT'] = ENSUREFLAG
                        if demandDisclosure:
                            datadict['DICT'] = "需求公示"
                        datadict['keyword'] = save
                        break
                item['title'] = saveTitle
                if outOfLoopCondition:
                    # print(f'当前标题为{datadict["title"]}属于CONCLUEDEDICT项目,进行保存')
                    continue
                # 进行中标数据的处理
                if WinTheBidOrNot:
                    money, sender, winBidder = self.singlePageAnalyse(winType=WinTheBidOrNot,
                                                                      text=res['content'])
                    datadict['winBidder'] = winBidder
                # 进行招标数据的处理
                else:
                    money, sender = self.singlePageAnalyse(winType=WinTheBidOrNot, text=res['content'])
                #
                datadict['money'] = money
                datadict['sender'] = sender
                datadict['url'] = url
                datadict['publishDate'] = self.timeConvert(datadict['publishDate'])
                '''!important
                    :description: 判断是否为设计咨询标类型数据,并保存设计咨询标数据
                '''
                for key in CONSTDATA.DESIGN_DICT:
                    if self.isInDict(patt=key,
                                     text=datadict['title']):
                        DESIGN_DICT_FLAG = True
                        if DESIGN_DICT_FLAG := True:
                            break
                    else:
                        DESIGN_DICT_FLAG = False

                # 如果是中标数据,进行保存
                if WinTheBidOrNot:
                    if DICTTYPE and DESIGN_DICT_FLAG:
                        sheetname = CONSTDATA.DESIGN_WIN_BID_DATA
                    elif DICTTYPE:
                        sheetname = DICT_WIN_BID_DATA
                    else:
                        sheetname = NOT_DICT_WIN_BID_DATA
                    datatemp = [datadict['DICT'], datadict['keyword'], datadict['districtName'],
                                datadict['title'],
                                datadict['sender'],
                                datadict['winBidder'], datadict['publishDate'], datadict['money'],
                                datadict['url']]
                    self.lock.acquire()
                    try:
                        alldatatemp = copy.deepcopy(datatemp)
                        dataAppend(data=datatemp, worrbook=self.config.file_path, sheetname=sheetname)

                        alldatatemp.append(res['content'])
                        dataAppend(data=alldatatemp,
                                   worrbook=self.config.file_path,
                                   sheetname=DEVELOP_WINNING_BID_DATA)
                    except Exception as e:
                        # print(e)
                        raise e
                    finally:
                        # 释放锁
                        self.lock.release()
                # 如果是招标数据,进行保存
                else:
                    if DICTTYPE and DESIGN_DICT_FLAG:
                        sheetname = CONSTDATA.DESIGN_WIN_BID_DATA
                    elif DICTTYPE:
                        sheetname = DICT_BIDDING_DATA
                    else:
                        sheetname = NOT_DICT_BIDDING_DATA
                    datatemp = [datadict['DICT'], datadict['keyword'], datadict['districtName'],
                                datadict['title'],
                                datadict['sender'],
                                datadict['publishDate'], datadict['money'], datadict['url']]
                    date_obj = datetime.strptime(datadict['publishDate'], '%Y-%m-%d').strftime('%d-%b-%y')
                    print(date_obj, type(date_obj))
                    super().executeInsertSJZXBSJ(
                        sql=f'''
                        (
                        '{datadict['DICT']}'
                        ,'{datadict['keyword']}'
                        ,'{datadict['districtName']}'
                        ,'{datadict['title']}'
                        ,'{date_obj}'
                        ,'{datadict['money']}'
                        ,'{datadict['url']}'
                        )
                        '''
                    )
                    self.lock.acquire()
                    try:
                        # 发布数据
                        alldatatemp = copy.deepcopy(datatemp)
                        dataAppend(data=datatemp, worrbook=self.config.file_path, sheetname=sheetname)

                        # 开发者数据
                        alldatatemp.append(res['content'])
                        dataAppend(data=alldatatemp, worrbook=self.config.file_path, sheetname=DEVELOP_BIDDING_DATA)
                    except Exception as e:
                        raise e
                    finally:
                        # 释放锁
                        self.lock.release()

    # 线程入口
    async def threadMain(self):
        import logging

        # 创建或获取一个 logger
        logger = logging.getLogger(__name__)

        # 设置日志级别
        logger.setLevel(logging.ERROR)

        # 创建一个文件处理器，将日志记录到 app.log 文件中
        file_handler = logging.FileHandler('app1.log', mode='w')
        file_handler.setLevel(logging.ERROR)

        # 创建一个格式化器
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        file_handler.setFormatter(formatter)

        # 将处理器添加到 logger 中
        logger.addHandler(file_handler)

        threadList = []
        for i in self.TYPEOFBIDDING:
            for j in self.AREATYOE:
                try:
                    thread = threading.Thread(target=self.ccgpMainAnalyse, args=(i, j))
                    threadList.append(thread)
                    thread.start()
                except Exception as e:
                    logger.exception("线程启动时发生了一个错误：", e)
                    raise e
        for threadsingle in threadList:
            threadsingle.join()
